package edu.unc.ils.nlp;

import java.io.FileInputStream;
import java.io.ObjectInputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

import maui.stemmers.PorterStemmer;

import org.apache.commons.collections15.Bag;
import org.apache.commons.collections15.bag.HashBag;

import com.aliasi.hmm.HiddenMarkovModel;
import com.aliasi.hmm.HmmDecoder;
import com.aliasi.tokenizer.RegExTokenizerFactory;
import com.aliasi.tokenizer.Tokenizer;
import com.aliasi.util.Streams;

public class TermFrequency 
{
	
	/**
	 * Given a string of text, returns the term frequencies. Limits by part-of-speech if present.
	 * @param text
	 */
	public Bag<String> getTermFrequencies(String text) throws Exception
	{
		PorterStemmer stemmer = new PorterStemmer();
		
		String model = "/Users/cwillis/dev/hive/hive-data/lingpipe/postagger/models/pos-en-general-brown.HiddenMarkovModel";
		FileInputStream fis = new FileInputStream(model);
		ObjectInputStream ois = new ObjectInputStream(fis);
		HiddenMarkovModel hmm = (HiddenMarkovModel) ois.readObject();
		Streams.closeInputStream(ois);
		HmmDecoder decoder = new HmmDecoder(hmm);
		
		Bag<String> terms = new HashBag<String>();
		String[] tokens = getTokens(text);
		String[] tags = decoder.firstBest(tokens);
		for (int i=0; i<tokens.length; i++)
		{
			String term = tokens[i];
			String tag = tags[i];
			String stemmedTerm = stemmer.stem(term);
			terms.add(stemmedTerm);
		}
		
		return terms;
	}
	
	
	public String[] getTokens(String text) {
		char[] cs = text.toCharArray();
		Tokenizer tokenizer = new RegExTokenizerFactory("(-|'|\\d|\\p{L})+|\\S").tokenizer(cs, 0, cs.length);
		
		return tokenizer.tokenize();
	}
	
}
